{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Perform Feature Selection using XGB Classifier\n",
    "Train XGBClassifier on features set 1- samples have 16 features. Then select features with top importance scores to form a new feature set. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  Get the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stored variables and their in-db values:\n",
      "X_16_val                  -> array([[ 0.10924883,  1.83030605, -0.14807631, ...\n",
      "X_32_val                  -> array([[ 0.66944195,  0.46536115,  0.79919788, ...\n",
      "X_32test_std              -> defaultdict(<class 'list'>, {0: array([[ 0.6694419\n",
      "X_32train_std             -> array([[-0.74031227,  0.0126481 , -0.30967801, ...\n",
      "X_test                    -> defaultdict(<class 'list'>, {0: array([[[ -6.40490\n",
      "X_test_std                -> defaultdict(<class 'list'>, {0: array([[ 0.1092488\n",
      "X_train                   -> array([[[ 0.00119031,  0.00873315,  0.00641749, ..\n",
      "X_train_std               -> array([[-0.74031227,  0.0126481 , -0.30967801, ...\n",
      "snrs                      -> [-20, -18, -16, -14, -12, -10, -8, -6, -4, -2, 0, \n",
      "y_16_val                  -> array([6, 6, 5, ..., 0, 4, 1])\n",
      "y_32_test                 -> defaultdict(<class 'list'>, {0: array([2, 2, 4, ..\n",
      "y_32_train                -> array([0, 3, 4, ..., 0, 3, 1])\n",
      "y_32_val                  -> array([2, 2, 4, ..., 0, 7, 3])\n",
      "y_test                    -> defaultdict(<class 'list'>, {0: array([6, 6, 5, ..\n",
      "y_train                   -> array([0, 3, 4, ..., 0, 3, 1])\n"
     ]
    }
   ],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from collections import defaultdict\n",
    "import xgboost as xgb\n",
    "from xgboost.sklearn import XGBClassifier\n",
    "from sklearn.cross_validation import train_test_split\n",
    "from sklearn.datasets import dump_svmlight_file\n",
    "from sklearn.externals import joblib\n",
    "from sklearn.metrics import precision_score\n",
    "from sklearn import cross_validation, metrics\n",
    "from time import time\n",
    "import pandas as pd\n",
    "import collections\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.feature_selection import SelectFromModel\n",
    "from numpy import sort\n",
    "\n",
    "%store -r\n",
    "%store"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training data:  (80000, 16) and labels:  (80000,)\n",
      " \n",
      "Test data:\n",
      "Total 20 (4000, 16) arrays for SNR values:\n",
      "[-20, -18, -16, -14, -12, -10, -8, -6, -4, -2, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18]\n"
     ]
    }
   ],
   "source": [
    "print(\"Training data: \", X_train_std.shape, \"and labels: \", y_train.shape)\n",
    "print(\" \")\n",
    "print(\"Test data:\")\n",
    "print(\"Total\", len(X_test_std), X_test_std[18].shape, \"arrays for SNR values:\")\n",
    "print(sorted(X_test_std.keys()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  Train and test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def train_and_test(model, train_data=X_train_std, train_labels=y_train, test_data=X_test_std, test_labels=y_test):\n",
    "    \n",
    "    dtrain = xgb.DMatrix(X_train_std, label=y_train)\n",
    "    dtest = defaultdict(list)\n",
    "    for snr in snrs:\n",
    "        dtest[snr] = xgb.DMatrix(X_test_std[snr], label=y_test[snr])\n",
    "    parameters = model.get_xgb_params()\n",
    "    \n",
    "    eval_history = xgb.cv(parameters, dtrain, metrics = 'mlogloss', nfold = 5, early_stopping_rounds = 20,\n",
    "                         num_boost_round = model.get_params()['n_estimators'])\n",
    "    model.set_params(n_estimators = eval_history.shape[0])\n",
    "    \n",
    "    eval_set = [(X_train_std, y_train), (X_test_std[10], y_test[10])]\n",
    "    start = time()\n",
    "    model.fit(train_data, train_labels, eval_metric = 'mlogloss', eval_set = eval_set) #fit model to data\n",
    "    print(\"Fitting model to data took {} minutes\".format(time() - start))\n",
    "    \n",
    "    # Compute metric for training data\n",
    "    train_predict = model.predict(train_data)\n",
    "    train_predict_prob = model.predict_proba(train_data)\n",
    "    print(\"Log loss on training set =\", metrics.log_loss(train_labels, train_predict_prob))\n",
    "    \n",
    "    # Compute metric for test data\n",
    "    test_logloss = []\n",
    "    for snr in snrs:\n",
    "        test_predict = model.predict(test_data[snr])\n",
    "        test_predict_prob = model.predict_proba(test_data[snr])\n",
    "        loss = metrics.log_loss(test_labels[snr], test_predict_prob)\n",
    "        test_logloss.append(loss)\n",
    "    print(\"Log loss on test set =\", np.mean(test_logloss))\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-mlogloss:1.95789\tvalidation_1-mlogloss:1.83349\n",
      "[1]\tvalidation_0-mlogloss:1.86774\tvalidation_1-mlogloss:1.65038\n",
      "Fitting model to data took 1.2433075904846191 minutes\n",
      "Log loss on training set = 1.86773986322\n",
      "Log loss on test set = 1.86907329542\n"
     ]
    }
   ],
   "source": [
    "xgb1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=2,\n",
    "        max_depth=5,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.8,\n",
    "        colsample_bytree=0.8,\n",
    "        objective= 'multi:softmax',\n",
    "        num_class = 8,\n",
    "        nthread=4,\n",
    "        scale_pos_weight=1,\n",
    "        seed=27)\n",
    "model = train_and_test(xgb1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Accuracy vs. SNR on test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test accuracy at  -20 dB = 0.12425\n",
      "Test accuracy at  -18 dB = 0.12525\n",
      "Test accuracy at  -16 dB = 0.1235\n",
      "Test accuracy at  -14 dB = 0.12725\n",
      "Test accuracy at  -12 dB = 0.153\n",
      "Test accuracy at  -10 dB = 0.1695\n",
      "Test accuracy at  -8 dB = 0.2575\n",
      "Test accuracy at  -6 dB = 0.334\n",
      "Test accuracy at  -4 dB = 0.3795\n",
      "Test accuracy at  -2 dB = 0.37775\n",
      "Test accuracy at  0 dB = 0.407\n",
      "Test accuracy at  2 dB = 0.5415\n",
      "Test accuracy at  4 dB = 0.73275\n",
      "Test accuracy at  6 dB = 0.77825\n",
      "Test accuracy at  8 dB = 0.79475\n",
      "Test accuracy at  10 dB = 0.811\n",
      "Test accuracy at  12 dB = 0.80825\n",
      "Test accuracy at  14 dB = 0.81025\n",
      "Test accuracy at  16 dB = 0.80325\n",
      "Test accuracy at  18 dB = 0.81125\n"
     ]
    }
   ],
   "source": [
    "predictions = defaultdict(list)\n",
    "accuracy = defaultdict(list)\n",
    "y_pred = defaultdict(list)\n",
    "   \n",
    "for snr in snrs:\n",
    "    y_pred[snr] = model.predict(X_test_std[snr])\n",
    "    predictions[snr] = [round(value) for value in y_pred[snr]]\n",
    "    accuracy[snr] = accuracy_score(y_test[snr], predictions[snr])\n",
    "    print (\"Test accuracy at \",snr,\"dB =\", accuracy[snr])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Importance score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Feature importance scores- highest first\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Features</th>\n",
       "      <th>Importance Scores</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>f14</td>\n",
       "      <td>0.194570</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>f5</td>\n",
       "      <td>0.158371</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>f6</td>\n",
       "      <td>0.115385</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>f13</td>\n",
       "      <td>0.115385</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>f10</td>\n",
       "      <td>0.095023</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>f9</td>\n",
       "      <td>0.090498</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>f1</td>\n",
       "      <td>0.065611</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>f11</td>\n",
       "      <td>0.036199</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>f15</td>\n",
       "      <td>0.036199</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>f12</td>\n",
       "      <td>0.029412</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>f4</td>\n",
       "      <td>0.020362</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>f2</td>\n",
       "      <td>0.018100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>f3</td>\n",
       "      <td>0.015837</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>f7</td>\n",
       "      <td>0.006787</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>f16</td>\n",
       "      <td>0.002262</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>f8</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Features  Importance Scores\n",
       "0       f14           0.194570\n",
       "1        f5           0.158371\n",
       "2        f6           0.115385\n",
       "3       f13           0.115385\n",
       "4       f10           0.095023\n",
       "5        f9           0.090498\n",
       "6        f1           0.065611\n",
       "7       f11           0.036199\n",
       "8       f15           0.036199\n",
       "9       f12           0.029412\n",
       "10       f4           0.020362\n",
       "11       f2           0.018100\n",
       "12       f3           0.015837\n",
       "13       f7           0.006787\n",
       "14      f16           0.002262\n",
       "15       f8           0.000000"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feature_names = ['f'+str(i) for i in range(1,17)]\n",
    "print(\"Feature importance scores- highest first\")\n",
    "df1 = pd.DataFrame(data = {'Features':feature_names, 'Importance Scores': model.feature_importances_})\n",
    "sorted_df = df1.sort_values(by = ['Importance Scores'], ascending = False)\n",
    "sorted_df.reset_index(inplace = True)\n",
    "del sorted_df['index']\n",
    "sorted_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAEKCAYAAAA4t9PUAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAH6lJREFUeJzt3X+UXVV99/H3pwEiVX5nbCkhJkoo4o8ihGBbixQKxEUl\n2IKEUn74qKk/0PZx6TK2NlTULqhtra1IRUBQ0YgoOpZgtCrYRwvNABEIEB1ihESehwgICPIj8Hn+\nOHvgMkxyz9wzZzLDfF5r3TXn7LP39+4zuTffOfucs49sExER0atf29odiIiIyS2JJCIiGkkiiYiI\nRpJIIiKikSSSiIhoJIkkIiIaSSKJiIhGkkgiIqKRJJKIiGhkm63dgfEwY8YMz549e2t3IyJiUrn2\n2mt/bruvW71WE4mkBcDHgGnAebbPHLb9XcCbgE3ARuB/2f5p2XYK8P5S9UO2LyrlBwAXAtsDy4G/\ndJd5XmbPns3AwMBY7VZExJQg6ad16rU2tCVpGnA28BpgX+AESfsOq3Y9MM/2y4FLgX8obXcFTgcO\nAuYDp0vapbQ5B3gzMLe8FrS1DxER0V2b50jmA4O219p+FFgGLOysYPu7th8qq1cDM8vykcC3bN9j\n+17gW8ACSbsDO9q+uhyFfAY4psV9iIiILtpMJHsAd3Ssry9lm/NG4Ioubfcoy3VjRkREyybEyXZJ\nfw7MA149hjEXA4sBZs2aNVZhIyJimDaPSDYAe3aszyxlTyPpj4C/AY62/UiXtht4avhrszEBbJ9r\ne57teX19XS86iIiIHrWZSFYCcyXNkbQdsAjo76wg6RXAJ6mSyF0dm1YAR0japZxkPwJYYftO4H5J\nr5Qk4GTgay3uQ0REdNHa0JbtTZJOo0oK04ALbK+WdAYwYLsf+AjwPOBLVV7gdttH275H0gepkhHA\nGbbvKctv46nLf6/gqfMqERGxFWgqPGp33rx5zn0kERGjI+la2/O61csUKRER0ciEuGorImJrmr3k\n8kbt15151Bj1ZHLKEUlERDSSRBIREY0kkURERCNJJBER0UgSSURENJJEEhERjSSRREREI0kkERHR\nSBJJREQ0kkQSERGNJJFEREQjSSQREdFIEklERDSSRBIREY0kkURERCNJJBER0UiriUTSAklrJA1K\nWjLC9oMlXSdpk6RjO8r/UNKqjtfDko4p2y6U9JOObfu1uQ8REbFlrT0hUdI04GzgcGA9sFJSv+2b\nO6rdDpwKvLuzre3vAvuVOLsCg8A3O6q8x/albfU9IiLqa/NRu/OBQdtrASQtAxYCTyYS2+vKtie2\nEOdY4ArbD7XX1YiI6FWbQ1t7AHd0rK8vZaO1CPjCsLIPS7pB0kclTe+1gxER0dyEPtkuaXfgZcCK\njuL3AfsABwK7Au/dTNvFkgYkDWzcuLH1vkZETFVtJpINwJ4d6zNL2Wi8HrjM9mNDBbbvdOUR4NNU\nQ2jPYPtc2/Nsz+vr6xvl20ZERF1tJpKVwFxJcyRtRzVE1T/KGCcwbFirHKUgScAxwE1j0NeIiOhR\na4nE9ibgNKphqVuAS2yvlnSGpKMBJB0oaT1wHPBJSauH2kuaTXVEc9Ww0BdLuhG4EZgBfKitfYiI\niO7avGoL28uB5cPKlnYsr6Qa8hqp7TpGODlv+9Cx7WVERDQxoU+2R0TExJdEEhERjSSRREREI0kk\nERHRSBJJREQ0kkQSERGNJJFEREQjSSQREdFIEklERDSSRBIREY0kkURERCNJJBER0UirkzbG1DR7\nyeWN2q8786gx6klEjIcckURERCNJJBER0UgSSURENJJEEhERjbSaSCQtkLRG0qCkJSNsP1jSdZI2\nSTp22LbHJa0qr/6O8jmSrikxv1ieBx8REVtJa4lE0jTgbOA1wL7ACZL2HVbtduBU4PMjhPiV7f3K\n6+iO8rOAj9reC7gXeOOYdz4iImpr84hkPjBoe63tR4FlwMLOCrbX2b4BeKJOQEkCDgUuLUUXAceM\nXZcjImK02kwkewB3dKyvL2V1PUfSgKSrJQ0li92AX9je1GPMiIgYYxP5hsQX2N4g6YXAdyTdCNxX\nt7GkxcBigFmzZrXUxYiIaPOIZAOwZ8f6zFJWi+0N5eda4ErgFcDdwM6ShhLgZmPaPtf2PNvz+vr6\nRt/7iIiopc1EshKYW66y2g5YBPR3aQOApF0kTS/LM4DfB262beC7wNAVXqcAXxvznkdERG2tJZJy\nHuM0YAVwC3CJ7dWSzpB0NICkAyWtB44DPilpdWn+YmBA0g+pEseZtm8u294LvEvSINU5k/Pb2oeI\niOiu1XMktpcDy4eVLe1YXkk1PDW83Q+Al20m5lqqK8IiImICyJ3tERHRSBJJREQ0kkQSERGNJJFE\nREQjtRKJpFdJekNZ7pM0p91uRUTEZNE1kUg6neqS2/eVom2Bz7XZqYiImDzqHJG8DjgaeBDA9s+A\nHdrsVERETB51Esmj5Y5yA0h6brtdioiIyaROIrlE0iep5rh6M/CfwKfa7VZEREwWXe9st/2Pkg4H\n7gd+G1hq+1ut9ywiIiaFLSaS8pTDFbb/CEjyiIiIZ9ji0Jbtx4GHJO00Tv2JiIhJps6kjQ8DN0r6\nFuXKLQDb72ytVxERMWnUSSSXl1dERMQz1DnZflF5MNXepWiN7cfa7VZEREwWXROJpEOAi4B1gIA9\nJZ1i+3vtdi0iIiaDOkNb/wQcYXsNgKS9gS8AB7TZsYiImBzq3JC47VASAbD9I6r5tiIiImolkgFJ\n50k6pLw+BQzUCS5pgaQ1kgYlLRlh+8GSrpO0SdKxHeX7SfpvSasl3SDp+I5tF0r6iaRV5bVfnb5E\nREQ76gxtvRV4OzB0ue9/AZ/o1qjczHg2cDiwHlgpqd/2zR3VbgdOBd49rPlDwMm2fyzpt4BrJa2w\n/Yuy/T22L63R94iIaFmdRLIN8DHb/wxPJojpNdrNBwZtry3tlgELgScTie11ZdsTnQ3L8NnQ8s8k\n3QX0Ab8gIiImlDpDW98Gtu9Y355q4sZu9gDu6FhfX8pGRdJ8YDvgto7iD5chr49KqpPUIiKiJXUS\nyXNs/3JopSz/entdeoqk3YHPAm+wPXTU8j5gH+BAYFeqh26N1HaxpAFJAxs3bhyP7kZETEl1EsmD\nkvYfWpF0APCrGu02AHt2rM8sZbVI2pHqjvq/sX31ULntO115BPg01RDaM9g+1/Y82/P6+vrqvm1E\nRIxSnXMkfwV8SdLPqG5I/E3g+C03AWAlMLc8330DsAj4szqdKnfSXwZ8ZvhJdUm7275TkoBjgJvq\nxIyIiHbUmSJlpaR9qJ5FAjWnSLG9SdJpwApgGnCB7dWSzgAGbPdLOpAqYewCvFbSB2y/BHg9cDCw\nm6RTS8hTba8CLpbUR5XUVgFvGc0OR0TE2KozRcpxwDds3yTp/cD+kj5k+7pubW0vB5YPK1vasbyS\nashreLvPAZ/bTMxDu71vRESMnzrnSP7W9gOSXgUcSTXv1jntdisiIiaLOonk8fLzKOAc21+juhw3\nIiKiViLZIOmTVCfYl5f7Nuq0i4iIKaBOQng91QnzI8sUJbsC72m1VxERMWnUuWrrIeArHet3Ane2\n2amIiJg86txHErHVzF7S7CnP6848aox6EhNNPhsTR851REREI7USiaQXSPqjsry9pB3a7VZEREwW\nXROJpDcDlwKfLEUzga+22amIiJg86hyRvB34feB+ANs/Bp7fZqciImLyqJNIHrH96NCKpG0At9el\niIiYTOokkqsk/TWwvaTDgS8BX2+3WxERMVnUSSRLgI3AjcBfUE3C+P42OxUREZNHnftItqeaAv5T\n8OQz27cHHmqzYxERMTm0+cz2iIiYAib0M9sjImLia/OZ7RERMQW0+cz2iIiYAroekZTH4e4DvJXq\n+egvtn1tneCSFkhaI2lQ0pIRth8s6TpJmyQdO2zbKZJ+XF6ndJQfIOnGEvNfJalOXyIioh11J208\nEHg5sD9wgqSTuzUoV3edDbwG2Le023dYtduBU4HPD2u7K3A6cBAwHzhd0i5l8znAm4G55bWg5j5E\nREQLug5tSfos8CJgFU89dtfAZ7o0nQ8M2l5b4iwDFgI3D1Wwva5se2JY2yOBb9m+p2z/FrBA0pXA\njravLuWfAY4Brui2HxER0Y4650jmAfvaHu20KHsAd3Ssr6c6wui17R7ltX6E8meQtBhYDDBr1qya\nbxsREaNVZ2jrJqoT7JOK7XNtz7M9r6+vb2t3JyLiWavOEckM4GZJ/wM8MlRo++gu7TYAe3aszyxl\ndWwADhnW9spSPrPHmBER0YI6ieTveoy9EpgraQ7Vf/aLgD+r2XYF8PcdJ9iPAN5n+x5J90t6JXAN\ncDLwbz32LyIixkDXRGL7ql4C294k6TSqpDCNar6u1ZLOAAZs90s6ELgM2AV4raQP2H5JSRgfpEpG\nAGcMnXgH3gZcSDVVyxXkRHtExFZV56qtV1L91f9iYDuqpPCg7R27tbW9nGq24M6ypR3LK3n6UFVn\nvQuAC0YoHwBe2u29IyJifNQZ2vo41bDUl6iu4DqZ6v6NiJiAZi+5vFH7dWceNUY9iamiTiLB9qCk\nabYfBz4t6Qct9ysiIiaJOonkIUnbAask/QNwJ/DcdrsVERGTRZ37SE4q9U4DHqS6pPdP2uxURERM\nHnWOSI6x/THgYeADAJL+EvhYmx2LmEpyXiMmszpHJKeMUHbqGPcjIiImqc0ekUg6geoGwhdK6u/Y\ntANwz8itIiJiqtnS0NYPqE6szwD+qaP8AeCGNjsVERGTx2YTie2fSlpPdfNhT3e3R0TEs98Wz5GU\n+0YekrTTOPUnIiImmTpXbT0M3FgeLvXgUKHtd7bWq4iImDTqJJLLyysiIuIZ6sz+e1G5s33vUrTG\n9mPtdisiIiaLOrP/HgJcBKwDBOwp6RTb32u3axERMRnUGdr6J+AI22sAJO0NfAE4oM2ORUTE5FDn\nzvZth5IIgO0fAdu216WIiJhM6hyRDEg6D/hcWT8RGGivSxERMZnUOSJ5K3Az8M7yurmUdSVpgaQ1\nkgYlLRlh+3RJXyzbr5E0u5SfKGlVx+sJSfuVbVeWmEPbnl9vVyMiog11rtp6RNLHgW8DT1BdtfVo\nt3aSpgFnA4cD64GVkvpt39xR7Y3Avbb3krQIOAs43vbFwMUlzsuAr9pe1dHuxPLI3YiI2Mq6HpFI\nOgq4jWra+I8Dg5JeUyP2fGDQ9tqSeJYBC4fVWUh1RRjApcBhkjSszgmlbURETEB1r9r6Q9uDAJJe\nRHWD4hVd2u0B3NGxvh44aHN1bG+SdB+wG/DzjjrH88wE9GlJjwNfBj5k2zX2IyIiWlDnHMldQ0mk\nWAvc1VJ/nkbSQcBDtm/qKD7R9suAPyivkzbTdrGkAUkDGzduHIfeRkRMTXUSyWpJyyWdKukU4OtU\n5zv+RNKWHrm7geqxvENmlrIR60jaBtgJuLtj+yKqe1aeZHtD+fkA8HmqIbRnsH2u7Xm25/X19XXb\nx4iI6FGdRPIc4P8BrwYOATYCuwKvBf54C+1WAnMlzSlTrCwC+ofV6eepJzAeC3xnaJhK0q8Br6fj\n/IikbSTNKMvblve/iYiI2GrqXLX1hl4Cl3MepwErgGnABbZXSzoDGLDdD5wPfFbSINVTFxd1hDgY\nuMP22o6y6cCKkkSmAf8JfKqX/kVExNioM9fWHOAdwOzO+raP7tbW9nJg+bCypR3LDwPHbabtlcAr\nh5U9SKZmiYiYUOpctfVVqiOHr1PdRxIREfGkWg+2sv2vrfckIiImpTqJ5GOSTge+CTwyVGj7utZ6\nFRERk0adRPIyqns1DuWpoS2X9YiImOLqJJLXAS+sM79WRERMPXUSyQ+BnRmnu9knmtlLen9c/boz\njxrDnkRETEx1EslvALdKWsnTz5F0vfw3IiKe/eokktNb70VERExade5sv2o8OhIREZPTZhOJpAeo\nrs56xibAtndsrVcRETFpbDaR2N5hPDsSERGTU53ZfyMiIjYriSQiIhpJIomIiEaSSCIiopEkkoiI\naCSJJCIiGkkiiYiIRlpNJJIWSFojaVDSkhG2T5f0xbL9GkmzS/lsSb+StKq8/r2jzQGSbixt/lWS\n2tyHiIjYstYSiaRpwNnAa4B9gRMk7Tus2huBe23vBXwUOKtj22229yuvt3SUnwO8GZhbXgva2oeI\niOiuzSOS+cCg7bXlWSbLgIXD6iwELirLlwKHbekIQ9LuwI62r7Zt4DPAMWPf9YiIqKvNRLIHcEfH\n+vpSNmId25uA+4DdyrY5kq6XdJWkP+iov75LzIiIGEd1ppHfGu4EZtm+W9IBwFclvWQ0ASQtBhYD\nzJo1q4UuRkQEtHtEsgHYs2N9ZikbsY6kbYCdgLttP2L7bgDb1wK3AXuX+jO7xKS0O9f2PNvz+vr6\nxmB3IiJiJG0mkpXAXElzJG0HLAL6h9XpB04py8cC37FtSX3lZD2SXkh1Un2t7TuB+yW9spxLORn4\nWov7EBERXbQ2tGV7k6TTgBXANOAC26slnQEM2O4Hzgc+K2kQuIcq2QAcDJwh6THgCeAttu8p294G\nXAhsD1xRXhERsZW0eo7E9nJg+bCypR3LDwPHjdDuy8CXNxNzAHjp2PY0IiJ6lTvbIyKikSSSiIho\nJIkkIiIaSSKJiIhGkkgiIqKRJJKIiGgkiSQiIhpJIomIiEaSSCIiopGJOvtvREwQs5dc3nPbdWce\nNYY9mTym2u8sRyQREdFIEklERDSSRBIREY0kkURERCNJJBER0UgSSURENJLLfyNi3Ey1y2KnihyR\nREREI60mEkkLJK2RNChpyQjbp0v6Ytl+jaTZpfxwSddKurH8PLSjzZUl5qryen6b+xAREVvW2tCW\npGnA2cDhwHpgpaR+2zd3VHsjcK/tvSQtAs4Cjgd+DrzW9s8kvRRYAezR0e7E8uz2iIjYyto8IpkP\nDNpea/tRYBmwcFidhcBFZflS4DBJsn297Z+V8tXA9pKmt9jXiIjoUZuJZA/gjo719Tz9qOJpdWxv\nAu4DdhtW50+B62w/0lH26TKs9beSNNKbS1osaUDSwMaNG5vsR0REbMGEPtku6SVUw11/0VF8ou2X\nAX9QXieN1Nb2ubbn2Z7X19fXfmcjIqaoNhPJBmDPjvWZpWzEOpK2AXYC7i7rM4HLgJNt3zbUwPaG\n8vMB4PNUQ2gREbGVtJlIVgJzJc2RtB2wCOgfVqcfOKUsHwt8x7Yl7QxcDiyx/f2hypK2kTSjLG8L\n/DFwU4v7EBERXbSWSMo5j9Oorri6BbjE9mpJZ0g6ulQ7H9hN0iDwLmDoEuHTgL2ApcMu850OrJB0\nA7CK6ojmU23tQ0REdNfqne22lwPLh5Ut7Vh+GDhuhHYfAj60mbAHjGUfo5I7jiOiVxP6ZHtEREx8\nSSQREdFIJm0cR02GjyBDSBExMeWIJCIiGskRySSWE+QRMRHkiCQiIhpJIomIiEaSSCIiopEkkoiI\naCSJJCIiGkkiiYiIRpJIIiKikSSSiIhoJIkkIiIayZ3tMaVkNoCIsZcjkoiIaCSJJCIiGmk1kUha\nIGmNpEFJS0bYPl3SF8v2ayTN7tj2vlK+RtKRdWNGRMT4au0ciaRpwNnA4cB6YKWkfts3d1R7I3Cv\n7b0kLQLOAo6XtC+wCHgJ8FvAf0rau7TpFjMi4lljMpzXa/OIZD4waHut7UeBZcDCYXUWAheV5UuB\nwySplC+z/YjtnwCDJV6dmBERMY7aTCR7AHd0rK8vZSPWsb0JuA/YbQtt68SMiIhxJNvtBJaOBRbY\nflNZPwk4yPZpHXVuKnXWl/XbgIOAvwOutv25Un4+cEVptsWYHbEXA4vL6m8Da8Z8JyszgJ8/y2ON\ndbypEGus402FWGMdbyrEGut4w2O9wHZft0Zt3keyAdizY31mKRupznpJ2wA7AXd3adstJgC2zwXO\n7bXzdUkasD3v2RxrrONNhVhjHW8qxBrreFMh1ljH6zVWm0NbK4G5kuZI2o7q5Hn/sDr9wCll+Vjg\nO64OkfqBReWqrjnAXOB/asaMiIhx1NoRie1Nkk4DVgDTgAtsr5Z0BjBgux84H/ispEHgHqrEQKl3\nCXAzsAl4u+3HAUaK2dY+REREd61OkWJ7ObB8WNnSjuWHgeM20/bDwIfrxNzKxnL4bKLGGut4UyHW\nWMebCrHGOt5UiDXW8XqK1drJ9oiImBoyRUpERDSSRDJKkt4p6RZJX5b035IekfTuhrEs6QZJN0r6\ngaTfaRjv3hJvlaQBSa9qEOvisn6gpE3lsu5eY22QdF/p1ypJS7u33ny/JB1S4qyWdNVoYw2L92BH\nv26S9LikXXuMdZmkr0v6YenbGxr06/IS7wZJ/yPppaNsv9nPad3phmrGukDSXeWS/kZ9k7SnpO9K\nurn8/v6yQaznlN/b0L/FB5rsZ6k3TdL1kv6jyX6WOuvK936VpIGGsXaWdKmkW0vd391S/8aU7bxG\n8QJuBeYAzwcOpDqP8+6GsX4P2KWUvQa4pmG85/HUsOXLgVt7jVWWpwHfoTo3dWyDfh0C/McY/f53\nproYY1Ypf36TeMPKXkt1BWGvfftr4KxS1kd1Icl2Pcb6CHB6KdsH+PZYfE7Lv+ltwAuB7YAfAvv2\n+pkHDgb2B24ag77tDuxflncAftRr3wABzyvL2wLXAK/sdT9LvXcBn+/2Wa75e1sHzGj6Oyt1LgLe\nVJa3A3Zu8l0bzStHJKMg6d+pvnj9wIm2VwKPjUGsg2zfWzZdTXV/TJN4b3b5NAHPBUZ1IqwzlqT/\nDbwD+DJwV8N+vWK07bcQ6+3AV2zfDmC7Ud/Kfg45AfhCg74Z2EGSqJL6PVRXH/YS621USRzbtwKz\nJf3GKNpv7nNaa7qhup95298r+zmafRsxnu07bV9Xlh8AbmGEGSxqxrLtX5bVbcvrGd+HuvspaSZw\nFHBe0/2sq04sSTtRJfPzAWw/avsXvbxfT8YrYz1bXgz7C4LqLvxej0ieFquUvRs4r2k84HVUf8Xc\nA/xur7GovsBXUQ2DXkhvRyRDsQ6huuH0h1QzFbykQax/oZrA80rgWuDkMfr3/PXyO9u1Qd92AL4L\n3An8EjiqQay/Bz5ayuZTJaQDmn5Oqe7bOq9j/STg400+88Bs6h2R1P4OlZi3Azv2Govq6GtV+bc4\nq0m/qOYEPIAaR9c14/0EuK58hhc3+Pfcj+peuwuB66kS3XN7+U708soRyQQi6Q+pZkR+b9NYti+z\nvQ9wDPDBBqH+BXiv7Sea9onqC/MC278D/Bvw1QaxtqH6Qh8FHAn8rZ6aIbqJ1wLft931r+stOJLq\nP67fovqCf1zSjj3GOhPYWdIqqiPD64HHG/Rt0pD0PKoj4b+yfX+vcWw/bns/qiP9+XXPM43Qnz8G\n7rJ9ba99GcGrbO9PNaT9dkkH9xhnG6qhxXNsvwJ4EBi3x2wkkUwQkl5O9VfEQtt3j1VcV0MOL5Q0\no8cQ84BlktZR/RX7CUnH9NiX+12GGVzdD7Rtg36tB1bYftD2z4HvAT1dpDDMIkY5rDWCN1ANu9n2\nINVfnfv0Eqj8zt5Q/iM8meqcy9qG/YN6UxhtNZK2pUoiF9v+yljEdDXU811gQY8hfh84unwXlgGH\nSvpcwz5tKD/vAi6jOursxXpgve1ryvqlVIllXCSRTACSZgFfAU6y/aMxiLdXGZ9H0v7AdKohpVGz\nPcf2bNuzqT6cb7Pd05GEpN/s6Nd8qs9fr0nza8CrJG0j6depJvu8pcdYQ/3bCXh1id3E7cBhJeZv\nUE0a2tN//uVKnO3K6puA7zX567zDhJ1uqHxGzgdusf3PDWP1Sdq5LG9P9SyjW3uJZft9tmeW78Ii\nqgsy/rxB354raYehZeAIoOtVb5vp2/8F7pD026XoMKqLUcZFq3e2P5tJ+k1gANgReELSX1FdWdLL\nl3wp1fT5nyj/z25ys0nY/hQ4WdJjwK+A410GUreyY4G3StpE1a9FvfbL9i2SvgHcADxBNd7f05ew\nw+uAb9p+sGGcDwIXSrqR6qqh95ajpl68GLhIkoHVVEOftW3pc6pRTjfUJdYXqM4bzJC0nupKs/N7\niUd1peFJwI1lSA/gr8tR7Ghj7U71+5tG9YfLJba7XbY7lt/tLfVtBnBZ+c5vA3ze9jca9O0dwMXl\nD4O1VEfG4yJ3tkdERCMZ2oqIiEaSSCIiopEkkoiIaCSJJCIiGkkiiYiIRpJIImpSNSPwqo7X7B5i\n7CzpbWPfu4itJ5f/RtQk6Ze2n9cwxmyqOZpGNU2HpGkuj5uOmGhyRBLRgKpnU3xE0kpVzwz5i1L+\nPEnflnSdqudNDM2seybwonJE8xFVz1T5j454H5d0alleJ2mppP8DHCfpRZK+IelaSf8laZ9S7zhV\nz1D5oaTvje9vICJ3tkeMxvYdd1r/xPbrqO40v8/2gZKmA9+X9E3gDuB15a7vGcDVkvqpJtJ7aZk7\nC0mHdHnPh22/qtT9NvAW2z+WdBDwCeBQqpkRjrS9YWg6kIjxlEQSUd+vhhJAhyOAl+upJ0fuBMyl\nmkTv78tsrk9QTce/xeeIbMYX4cmZcH8P+FKZUgOqOdQAvk81JcslVHO2RYyrJJKIZgS8w/aKpxVW\nw1N9VM8OeazMGPucEdpv4ulDzMPrDM379WvAL0ZIZNh+SzlCOQpYJWm/sZxBOqKbnCOJaGYF1USU\n2wJI2rvM5LoT1bMrHlP1nJkXlPoPUD34ashPgX0lTS/DUoeN9CZlUr6fSDquvI8k/U5ZfpHta2wv\nBX7O06eHj2hdjkgimjmP6il+15XpzzdSPUzsYuDrkgaoHnJ1K4DtuyV9X9JNwBW231OGpG6gejb5\n9Vt4rxOBcyS9n+qRscuonjb5EUlzqY6Ovl3KIsZNLv+NiIhGMrQVERGNJJFEREQjSSQREdFIEklE\nRDSSRBIREY0kkURERCNJJBER0UgSSURENPL/AX2Ch9BK4N/EAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x23416c062e8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scores using XGBoost's function:\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x2341db5cb00>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAEWCAYAAABi5jCmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XmUVPWd/vH3wxJFEFERFQggAVqlUVwi8cRBiAMu6AjG\nUQkZ93FLXCbBSGKikp85IUaDmnGScQ1RI3FBMeqYOAjRGIlCbERB1NGOgAqKGzStNvD5/VEXrGq7\nugvo6lvd9bzOqUPVrVv3fvpr2Z++23MVEZiZmW3ULu0CzMystLgxmJlZDjcGMzPL4cZgZmY53BjM\nzCyHG4OZmeVwYzArkKRfS/pR2nWYFZt8HYMVm6RqYFdgfdbkQRHx5lYscwRwR0T03rrqWidJvwGW\nRcQP067F2h5vMVhLOSYiumQ9trgpNAdJHdJc/9aQ1D7tGqxtc2OwVEn6iqS/SvpA0oJkS2Dje6dJ\nWixptaTXJJ2dTO8M/A/QU9Ka5NFT0m8kXZn1+RGSlmW9rpZ0iaTngRpJHZLP3SfpHUmvS7qgkVo3\nLX/jsiV9T9JKSW9JGivpKEkvS3pP0g+yPnuFpHsl/T75ef4uad+s9/eSNCcZhxcl/Uu99f5K0iOS\naoAzgAnA95Kf/Q/JfJMk/V+y/EWSxmUt41RJf5F0taT3k5/1yKz3d5J0m6Q3k/cfyHrvaElVSW1/\nlbRPwf+BrVVyY7DUSOoFPAxcCewETATuk7RLMstK4GigK3AaMFXS/hFRAxwJvLkFWyDjgTFAN2AD\n8AdgAdALOAy4SNLhBS5rN2Db5LOXATcB3wQOAP4J+JGkPbLmPxa4J/lZfwc8IKmjpI5JHX8CegDn\nA3dKqsj67DeAnwDbA78F7gSuSn72Y5J5/i9Z7w7AZOAOSbtnLWMYsAToDlwF3CJJyXu3A9sBg5Ma\npgJI2g+4FTgb2Bn4b+BBSdsUOEbWCrkxWEt5IPmL84Osv0a/CTwSEY9ExIaIeAyYBxwFEBEPR8T/\nRcafyfzi/KetrOP6iFgaEbXAl4FdIuLHEfFpRLxG5pf7SQUuqw74SUTUAdPJ/MK9LiJWR8SLwCJg\n36z550fEvcn8vyDTVL6SPLoAU5I6HgceItPENpoZEU8l4/RxQ8VExD0R8WYyz++BV4CDsmb5R0Tc\nFBHrgWnA7sCuSfM4EjgnIt6PiLpkvAHOAv47Iv4WEesjYhrwSVKztVGtdj+rtTpjI+J/603rC/yr\npGOypnUEZgMkuzouBwaR+SNmO2DhVtaxtN76e0r6IGtae+DJApe1KvklC1Cb/Lsi6/1aMr/wP7fu\niNiQ7ObqufG9iNiQNe8/yGyJNFR3gySdDHwH6JdM6kKmWW30dtb61yYbC13IbMG8FxHvN7DYvsAp\nks7PmvaFrLqtDXJjsDQtBW6PiH+v/0ayq+I+4GQyfy3XJVsaG3d9NHQ6XQ2Z5rHRbg3Mk/25pcDr\nETFwS4rfAl/c+ERSO6A3sHEX2BcltctqDn2Al7M+W//nzXktqS+ZrZ3DgKcjYr2kKj4br8YsBXaS\n1C0iPmjgvZ9ExE8KWI61Ed6VZGm6AzhG0uGS2kvaNjmo25vMX6XbAO8A65Kth9FZn10B7Cxph6xp\nVcBRyYHU3YCLmlj/M8Dq5IB0p6SGSklfbrafMNcBko5Lzoi6iMwumbnA34C1ZA4md0wOwB9DZvdU\nPiuA/lmvO5NpFu9A5sA9UFlIURHxFpmD+f8lacekhuHJ2zcB50gapozOksZI2r7An9laITcGS01E\nLCVzQPYHZH6hLQUuBtpFxGrgAuBu4H0yB18fzPrsS8BdwGvJcYueZA6gLgCqyRyP+H0T619P5uD2\nUOB14F3gZjIHb4thJnAimZ/n34Djkv35n5JpBEcmNfwXcHLyM+ZzC7D3xmM2EbEIuAZ4mkzTGAI8\ntRm1/RuZYyYvkTnofxFARMwD/h34z6TuV4FTN2O51gr5AjezFiDpCmBARHwz7VrMmuItBjMzy+HG\nYGZmObwryczMcniLwczMcrTK6xi6desWAwYMSLuMklNTU0Pnzp3TLqMkeWzy89jk15bGZv78+e9G\nxC5Nz9lKG8Ouu+7KvHnz0i6j5MyZM4cRI0akXUZJ8tjk57HJry2NjaR/FDqvdyWZmVkONwYzM8vh\nxmBmZjncGMzMLIcbg5mZ5XBjMDOzHG4MZmaWw43BzMxyuDGYmaVs6tSpDB48mMrKSsaPH8/HH39M\nRHDppZcyaNAg9tprL66//voWq6eoVz5LugA4l8xN0XsC+wOXRsTVyfvbAk+QuVNXB+DeiLi8mDWZ\nmZWS5cuXc/3117No0SI6derECSecwPTp04kIli5dyksvvUS7du1YuXJli9VU7EiM88jclaqGzE3F\nx9Z7/xPgaxGxRlJH4C+S/ici5ha5LjOzkrFu3Tpqa2vp2LEja9eupWfPnvzwhz/kd7/7He3aZXbs\n9OjRo8XqKVpjkPRrMvekfRC4NSKmShqTPU9kMr/XJC87Jo8mc8Br69bTb9LDzVxx6/fdIes41ePS\nII9Nfh6b/IoxNtVTcn4N0qtXLyZOnEifPn3o1KkTo0ePZvTo0YwfP57f//733H///eyyyy5cf/31\nDBw4sFlryadojSEizpF0BDAyIt7NN5+k9sB8YABwQ0T8Lc98ZwFnAXTvvguXDVlXhKpbt107Zb7I\n9nkem/w8NvkVY2zmzJmT83r16tVMmzaNO+64gy5dunDFFVdw6aWXsnbtWpYvX87VV1/NE088wde/\n/vWWO84QEUV7kLkpe/es11cAE/PM2w2YDVQ2tdxBgwaFfd7s2bPTLqFkeWzy89jk1xJjc/fdd8fp\np5++6fW0adPi3HPPjYqKinjttdciImLDhg3RtWvXrVoPMC8K/N1dMmclRcQHZBrDEWnXYmbWUvr0\n6cPcuXNZu3YtEcGsWbPYa6+9GDt2LLNnzwbgz3/+M4MGDWqxmlK9H4OkXYC6iPhAUidgFPCzNGsy\nM2tJw4YN4/jjj2f//fenQ4cO7Lfffpx11lnU1tYyYcIEpk6dSpcuXbj55ptbrKYWaQySdgPmAV2B\nDZIuAvYGdgemJccZ2gF3R8RDLVGTmVmpmDx5MpMnT86Zts022/Dww+mcFFDUxhAR/bJe9m5glueB\n/YpZg5mZbZ6SOcZgZmalwY3BzMxyuDGYmVkONwYzK2lLlixh6NChmx5du3bl2muv5b333mPUqFEM\nHDiQUaNG8f7776ddapuRSmOQdIGkxZLuTF5/WdI6ScenUY+Zla6Kigqqqqqoqqpi/vz5bLfddowb\nN44pU6Zw2GGH8corr3DYYYcxZcqUtEttM9LaYjgPOCoiJiSnqv4M+FNKtZhZKzFr1iy+9KUv0bdv\nX2bOnMkpp5wCwCmnnMIDDzyQcnVtR4tf4JYdrifpVjKhefcBXy50GQ7Ra5jD0PLz2ORXSmNTP2Cu\nvunTpzN+/HgAVqxYwe677w7AbrvtxooVK4peX7lo8S2GiDgHeBMYCdwNjAN+1dJ1mFnr8umnn/Lg\ngw/yr//6r597TxKSUqiqbUo1EgO4FrgkIjY09R/V6apNc0pmfh6b/EppbOonj2b7y1/+wh577MHi\nxYtZvHgxXbt25b777mPnnXdm1apVbL/99o1+fkusWbOm2ZfZGigTutfCK5WqgQOBZ4GNHaE7sBY4\nKyIa3VlYUVERS5YsKWqNrdGcOXMYMWJE2mWUJI9Nfq1lbE466SQOP/xwTjvtNAAuvvhidt55ZyZN\nmsSUKVN47733uOqqq5p1na1lbAohaX5EHFjIvKmerhoRe0REvyQ6417gvKaagpmVn5qaGh577DGO\nO+64TdMmTZrEY489xsCBA/nf//1fJk2alGKFbUvau5LMzJrUuXNnVq1alTNt5513ZtasWSlV1Lal\n0hjqhettnHZqy1diZmb1+cpnMzPL4cZgZmY53BjMzCyHG4OZmeVwYzArQf369WPIkCEMHTqUAw/M\nnHp+zz33MHjwYNq1a8e8efNSrtDasqI1hqwE1fskPS3pE0kTs97/oqTZkhZJelHShcWqxaw1mj17\nNlVVVZuaQGVlJTNmzGD48OEpV2ZtXTFPVz0POBKoAfoCY+u9vw74bkT8XdL2wHxJj0XEoiLWZNZq\n7bXXXmmXYGWiKI0hO0EVuDUipkrKiU2MiLeAt5LnqyUtBnoBTTYGp6s2rJRSMktNKY9NQ4mikhg9\nejSSOPvssznrrLNSqMzKVVEaQ0ScI+kIYGREvNvU/JL6AfsBf2tkHofoNaGUwtBKTSmPTUMhbVdd\ndRW77LIL77//PhMnTqS2tpZ9990XgA8++ID58+ezZs2aZll/uQbFFaJcxyb1SAxJXcjcj+GiiPgo\n33wRcSNwI2RC9M6fcGwLVdh6zJkzhxPaSOBXc2vNY7NgwQLq6uo2hbl169aNAw44YNNB6a3VloLi\nmlu5jk2qZyVJ6kimKdwZETPSrMWsVNTU1LB69epNz//0pz9RWVmZclVWTlJrDMrcgOEWYHFE/CKt\nOsxKzYoVKzjkkEPYd999OeiggxgzZgxHHHEE999/P7179+bpp59mzJgxHH744WmXam1U0XclSdoN\nmAd0BTZIugjYG9gH+DdgoaSqZPYfRMQjxa7JrJT179+fBQsWfG76uHHjGDduXAoVWbkpWmOol6Da\nu4FZ/sJnN+kxM7MS4SufzcwshxuDmZnlcGMwM7McbgxmZpbDjcGsBDld1dKUypXPki4AziVzCmsX\n4PXkrRkR8eM0ajIrNbNnz6Z79+6bXm9MVz377LNTrMrKQVqRGBuTV/sCEyPi6M35sEP0GlbKQXFp\nK+WxaShEryFOV7WW0uK7kuolr+7X0us3aw02pqsecMAB3HjjjWmXY2WmxbcYspNXgUrgh5IWAG+S\n2Xp4saHPOV21aaWcIJq2Uh4bp6uWrnIdm7TTVf8O9I2INZKOAh4ABjY0o9NVm9aaE0SLrTWPjdNV\n01OuY5PqWUkR8VFErEmePwJ0lNS9iY+ZtWlOV7W0pR27vVuSsoqkg5J6VqVZk1nanK5qaUt7V9Lx\nwLmS1gG1wEkRESnXZJYqp6ta2lJpDFnJq/+ZPMzMrET4ymczM8vhxmBmZjncGMzMLIcbg1ke69ev\nZ7/99uPoozOJLRMmTKCiooLKykpOP/106urqUq7QrDhSaQySLpC0WNLDku6X9LykZyT5ZG0rGddd\nd11OPtGECRN46aWXWLhwIbW1tdx8880pVmdWPGltMZwHHAUsAqoiYh/gZOC6lOoxy7Fs2TIefvhh\nzjzzzE3TjjrqKCQhiYMOOohly5alWKFZ8bT46ar1QvT6A0cARMRLkvpJ2jUiVjS2DKerNqyUE0TT\n1tjYNJRuetFFF3HVVVdtugI5W11dHbfffjvXXee/Y6xtSjtE7zvAccCTyZXPfYHewOcag0P0mlbK\nQXFpa2xs6oekPf3009TV1bF69WqqqqpYtWpVzjxXX301/fv3Z/369W0iYK1cg+IKUa5jk/aVz1OA\n6yRVAQuB54D1Dc2YHaLXp/+AuGZh2qWXnu8OWYfHpWGNjU31hBE5r//4xz8yf/58Tj31VD7++GM+\n+ugjbr75Zu644w4mT55Mhw4duPvuu2nXrm2cu1GuQXGFKNuxiYgWfwDVQPd605RM79rU5wcNGhT2\nebNnz067hJK1pWMze/bsGDNmTERE3HTTTXHwwQfH2rVrm7Gy9Pl7k19bGhtgXhT4OzrtEL1ukr6Q\nvDwTeCIiPkqzJrN8zjnnHFasWMHBBx/M0KFD+fGPfRdaa5vS3u+wFzBNUgAvAmekXI9ZjhEjRmza\nlbBunY/fWHlIO0TvXWBQGjWYmVnD2sbRMzMzazZuDGZmlsONwczMcrgxmJlZDjcGK2v1E1Rff/11\nhg0bxoABAzjxxBP59NNPU67QrOWlna56v6Q/SFog6UVJp6VRj5Wv+gmql1xyCf/xH//Bq6++yo47\n7sgtt9ySYnVm6Ug7XfVZYFFE7AuMAK7JuuDNrKjqJ6hGBI8//jjHH388AKeccgoPPPBAmiWapSLt\ndNXfAdtLEtAFeA9o8ioip6s2zOmq+f3miM6fm1Y/QXXVqlV069aNDh0y/1v07t2b5cuXt2idZqUg\n7XTVT8g0iDeB7YETI2JDQ59zumrTnK6aX/2UzIYSVJ966ilqa2s3zbdy5UpqamrafLpmuSaIFqJs\nx6bQUKXmfJCE6AHHA1PJBOgNAF7HIXpbrC0FfjW3+mMzadKk6NWrV/Tt2zd23XXX6NSpU3zjG9+I\nnXfeOerq6iIi4q9//WuMHj06hWpblr83+bWlsaG1hOgBpwEzkrpfJdMY9ky5JisDP/3pT1m2bBnV\n1dVMnz6dr33ta9x5552MHDmSe++9F4Bp06Zx7LHHplypWctLuzG8ARwGIGlXoAJ4LdWKrKz97Gc/\n4xe/+AUDBgxg1apVnHGGcx2t/KSdrvr/gN9IWkhmd9IlEfFuyjVZmclOUO3fvz/PPPNMugWZpSzt\ndFWA0WnUYGZmDUt7V5KZmZUYNwYzM8ux2Y1B0o6S9ilGMWZmlr6CGoOkOZK6StoJWADcJukXxS3N\nWrulS5cycuRI9t57bwYPHsx1110HwIIFCzj44IMZMmQIxxxzDB995Nt8m5WSQrcYdoiIj4DjgNsi\n4gDgn7d0pVkhendKul7Sq5Kel7T/li7TSk+HDh245pprWLRoEXPnzuWGG25g0aJFnHnmmUyZMoWF\nCxcybtw4fv7zn6ddqpllKbQxdJC0O3AC8FAzrHdjiN6dwMDkcRbwq2ZYtpWI3Xffnf33z/T67bff\nnr322ovly5fz8ssvM3z4cABGjRrFfffdl2aZZlZPoaer/hj4I/BURDwrqT/wypassF6I3iDg1ORy\n7bmSuknaPSLeamwZDtFrWNohetVTxuR/r7qa5557jmHDhjF48GBmzpzJ2LFjueeee1i6dGkLVmlm\nTVHmd3ILr1SqBg4EfgNMiYi/JNNnkbnIbV4Dn8kO0TvgsmtvarF6W4tdO8GK2vTWP6TXDg1Or62t\n5cILL+Sb3/wmw4cP54033uCXv/wlH374IV/96leZMWMGM2fOLGpta9asoUuXLkVdR2vlscmvLY3N\nyJEj50fEgYXMW9AWg6RBZHbz7BoRlclZSf8SEVduRZ2bJSJuBG4E6NN/QFyzMO2LtkvPd4esI81x\nqZ4w4nPT6urqOProoznnnHP4zne+s2n6ySefDMDLL7/Miy++uOnK42KZM2dO0dfRWnls8ivXsSn0\nt8hNwMXAfwNExPOSfgdsbWNYDnwx63XvZFqjOnVsz5JGdluUqzlz5jT4yzktEcEZZ5zBXnvtldMU\nVq5cSY8ePdiwYQNXXnkl55xzTopVmll9hR583i4i6gfINEfw/4PAycr4CvBhU8cXrPV46qmnuP32\n23n88ccZOnQoQ4cO5ZFHHuGuu+5i0KBB7LnnnvTs2ZPTTvMdXc1KSaFbDO9K+hIQAJKOB5rjF/gj\nZM5OehVYSyaG29qIQw45hHzHsC688MIWrsbMClVoY/gWmf37e0paTua+CRO2dKX1QvS+taXLMTOz\n5tdkY5DUDjgwIv5ZUmegXUSsLn5pZmaWhiaPMUTmHszfTp7XuCmYmbVthR58fkzSRElflLTTxkdR\nKzMzs1QUeozh9OTf7OMBQeYKZjMza0MK2mKIiD0aeLgptKDTTz+dHj16UFlZmTP9l7/8JXvuuSeD\nBw/m17/+dUrVmVlbUmjs9skNPQr43MYU1fskPS3pE0kT681zhKQlScLqpC39Qdq6U089lUcffTRn\n2uzZs5k5cyYLFizgxRdf5MQTT0ypOjNrSwrdlfTlrOfbAocBfwd+28TnzgOOBGqAvsDY7DcltQdu\nAEYBy4BnJT0YEYsKrKtsDB8+nOrq6pxpv/rVr5g0aRLbbLMNADvuuGMKlZlZW1NQY4iI87NfS+oG\nTGvsM/VSVG+NiKmS6udYHAS8GhGvJZ+ZDhwLNNoY2nq6amMppdlefvllnnzySS699FK23XZbxo8f\nX5a5LmbWvLY0ca2GTGR2XhFxjqQjgJER8W6e2XoB2ZnLy4BhDc1YL12Vy4Y0RyJHaZozZ06D099+\n+21qamo2vf/hhx+ycOFCpkyZwksvvcTll19ORUUFklqu2FZizZo1ece13Hls8ivXsSk0XfUPJHEY\nZI5L7A3cU6yiGpKdrlpRURHnTzi2JVdfEqqrq+ncufOmrYKKigrOP/98Ro4cyciRI7nyyiuprKxk\nl112SbfQElSuKZmF8NjkV65jU+gWw9VZz9cB/4iIZc2w/i1KV7WMsWPHMnv2bEaOHMnLL79MXV0d\n3bt3T7ssM2vlCr3A7aiI+HPyeCoilkn6WTOs/1lgoKQ9JH0BOInMMQmrZ/z48Rx88MEsWbKE3r17\nc8stt3D66afz2muvUVlZyUknncSkSZO8G8nMtlqhWwyjgEvqTTuygWkNkrQbMA/oCmyQdBGwd0R8\nJOnbZG4b2p7MQeoXC6yprNx1110NTr/jjjs2PS/HfaFm1vwabQySziVzyml/Sc9nvbU98FRTC6+X\noto7zzyPkInfNjOzEtDUFsPvgP8BfgpkX3y2OiLeK1pVZmaWmkYbQ0R8CHwIjAeQ1IPMBW5dJHWJ\niDeKX6KZmbWkQiMxjpH0Cpkb9PwZqCazJWFmZm1MoWclXQl8BXg5IvYgE4nR5DEGMzNrfQptDHUR\nsQpoJ6ldRMwGhhaxLqvH6apm1lIKbQwfSOoCPAncKek6Mhe6NaqQdNVkvvaSnpP00OaVXz6crmpm\nLaXQ6xiOBWqBi4AJwA7Ajwv4XKPpqlkuBBaTuc6hSeUYoud0VTNrKYXeqKeGTHTFiIiYBtwMfNrY\nZ+qlq06IiGeBugbm6w2MSZZpm2FjuuqwYcM49NBDeemll9IuyczagEJD9P6dTLLpTsCXyKSi/prM\nQegGFZiuCnAt8D0yF801VoPTVZ2uusXKNSWzEB6b/Mp2bCKiyQdQBXwBeC5r2sICPlcNdM96fQUw\nMev10cB/Jc9HAA8VUs+gQYOiHL3++usxePDgTa8PP/zwePzxxze97tmzZ6xcuTKN0kre7Nmz0y6h\nZHls8mtLYwPMiwJ+v0ZEwQefP4mITbuOJHXgsxjurfFV4F8kVQPTga9JuqPxj9hGG9NVAaermlmz\nKbQx/FnSD4BOkkaRuRfDH7Z25RHx/YjoHZlMpZOAxyPim1u73LbI6apm1lIKPStpEnAGsBA4m0zo\nXcEHixtLV928csuX01XNrKU0la7aJyLeiIgNwE3Jo2BRQLpq1rxzgDmbs3wzM2t+Te1KemDjE0n3\nFbkWMzMrAU01huwd1v2LWYiZmZWGphpD5HluZmZtVFONYV9JH0laDeyTPP9I0mpJPnC8lRoKxvvR\nj37EPvvsw9ChQxk9ejRvvvlmihWaWTlqtDFERPuI6BoR20dEh+T5xteN5hoVEqAn6VZJKyW90Bw/\nTGvTUDDexRdfzPPPP09VVRVHH300P/5xIZFUZmbNp9DTVbdEIQF6vwH+E/htEesoWQ0F43Xt+lm/\nramp8XUJZtbiitIY6gXo3RoRUyV9LjI0Ip6Q1G9zl99a01UbSk1tyKWXXspvf/tbdthhh01XNpuZ\ntRRlIjSKsOBMzMWBkQToSboCWBMRV9ebrx+ZjKRKGlEvRO+Ay67drEsqSsKQXjt8btrbb7/N97//\nfW677bbPvXfnnXfy6aefctpppxW0/DVr1tClS5etrrMt8tjk57HJry2NzciRI+dHxIGFzFvMXUnN\nKiJuBG4E6NN/QFyzsNWUvkn1hBGfn1ZdTefOnRkx4vPv9e/fn6OOOopp06YVtPw5c+Y0uBzz2DTG\nY5NfuY5N6/vtCnTq2J4lBe6WaW1eeeUVBg4cCMDMmTPZc889U67IzMpNq2wMbcX48eOZM2cO7777\nLr1792by5Mk88sgjLFmyhHbt2tG3b1/fx9nMWlzRG0NjAXqS7iJzH4bukpYBl0fELcWuqVQ0FIx3\nxhlnpFCJmdlnitYYCgnQi4jxxVq/mZltmULvx2BmZmXCjcHMzHK4MZiZWQ43BjMzy+HG0Eyuu+46\nKisrGTx4MNdee23a5ZiZbbFUGkNW8ur7kp6XVCVpnqRD0qhna73wwgvcdNNNPPPMMyxYsICHHnqI\nV199Ne2yzMy2SFoXuG1MXn0HqImIkLQPcDfQ5KW+aYboNRSEt3jxYoYNG8Z2220HwKGHHsqMGTP4\n3ve+19LlmZlttRbfYqiXvPrv8VmKX2da6V3iKisrefLJJ1m1ahVr167lkUceYenSpWmXZWa2RYqW\nrtroSrOSVyWNA34K9ADGRMTTeT5TEumqDSWkAjz88MPMnDmTTp060a9fPzp27Mi3v/3tFq2tLSVB\nNjePTX4em/za0thsTrpq6o0ha9pw4LKI+OemPl9RURFLliwpYoVb5wc/+AG9e/fmvPPOa9H1lmsS\nZCE8Nvl5bPJrS2MjqfXFbic37ekvqXt2w2gtVq5cSY8ePXjjjTeYMWMGc+fOTbskM7MtkmpjkDQA\n+L/k4PP+wDbAqjRr2lJf//rXWbVqFR07duSGG26gW7duaZdkZrZF0t5i+DpwsqQ6oBY4MdLYt9UM\nnnzyybRLMDNrFqk0hqzk1Z8lDzMzKxG+8tnMzHK4MZiZWQ43BjMzy+HGsIWmTp3K4MGDqaysZPz4\n8Xz88cdpl2Rm1izSDtG7T9LTkj6RNDGNWrbE8uXLuf7665k3bx4vvPAC69evZ/r06WmXZWbWLNIO\n0asB+gJjU6pji61bt47a2lo6duzI2rVr6dmzZ9olmZk1ixZvDPVC9G6NiKmSPh9Z2oiWTletn6ja\nq1cvJk6cSJ8+fejUqROjR49m9OjRLVaPmVkxlURWkqQrgDURcXUjn0ktRK9+cN7q1au5/PLLueyy\ny+jSpQtXXHEFhx56KKNGjWqxmhrSlgK/mpvHJj+PTX5taWw2J0Qv7SufCxYRNwI3AvTpPyCuWdhy\npVdPGJHz+p577mG//fZj7NjMHrA333yTuXPnph621ZYCv5qbxyY/j01+5To2raYxZOvUsT1LGrhh\nTkvp06dcib1NAAAKnElEQVQPc+fOZe3atXTq1IlZs2Zx4IEFNWIzs5Ln01W3wLBhwzj++OPZf//9\nGTJkCBs2bOCss85Kuywzs2aRdrrqbsA8oCuwQdJFwN4R8VGadRVi8uTJTJ48Oe0yzMyaXdohegC9\n06jBzMwa5l1JZmaWw43BzMxyuDGYmVkONwYzM8vhxlCgJUuWMHTo0E2Prl27cu2116ZdlplZs0vl\nrCRJFwDnAnsCCwEBq4FzI2JBGjU1paKigqqqKgDWr19Pr169GDduXMpVmZk1v7TTVXcHFkfE+5KO\nJBN5MSylmgo2a9YsvvSlL9G3b9+0SzEza3alkK761+StuRR4TUOx01Xrp6nWN336dMaPH1+09ZuZ\npakk0lWTaROBPSPizDyfabF01fppqtnq6uo4/vjjue2229hpp52KVsOWaEtJkM3NY5Ofxya/tjQ2\nrS5dVdJI4AzgkHzzZKerVlRUxPkTjm2h6nLNnDmTYcOGcdxxx6Wy/saUaxJkITw2+Xls8ivXsUm9\nMUjaB7gZODIiVqVdT1Puuusu70YyszYt1dNVJfUBZgD/FhEvp1lLIWpqanjsscdKcmvBzKy5pL3F\ncBmwM/BfkgDWFboPLA2dO3dm1aqS36gxM9sqaaernpk8zMysRPjKZzMzy+HGYGZmOdwYzMwsR1k1\nho8//piDDjqIfffdl8GDB3P55ZenXZKZWclJpTFIukDSYkl3ShohqUrSi5L+XMz1brPNNjz++OMs\nWLCAqqoqHn30UebOnVvMVZqZtTpph+i9D/wVOCIi3pDUo5grlbTp8va6ujrq6upITpM1M7NE2iF6\n04EZEfEGQESsLGQZhYboNRSGt379eg444ABeffVVvvWtbzFsWMmHuZqZtagW35UUEecAbwIjgV2A\nHSXNkTRf0snFXn/79u2pqqpi2bJlPPPMM7zwwgvFXqWZWauSaroqcEXy72FAJ+BpYExD8Rhbkq7a\nWEoqwLRp09h222058cQTN+8HKFFtKQmyuXls8vPY5NeWxmZz0lXTbgxnAp0i4vJk+i3AoxFxT2Of\nr6ioiCVLlmz2et955x06duxIt27dqK2tZfTo0VxyySUcffTRm/9DlKByTYIshMcmP49Nfm1pbCS1\nmtjtmcB/SuoAfIHM3dumFmtlb731Fqeccgrr169nw4YNnHDCCW2mKZiZNZdUG0NELJb0KPA8sAG4\nOSKKttN/n3324bnnnivW4s3M2oS0Q/SIiJ8DP0+jDjMz+7yyuvLZzMya5sZgZmY53BjMzCyHG4OZ\nmeVok43h9NNPp0ePHlRWVqZdiplZq1O0xpCVoHqfpKclfSJpYr15qiUtTNJV5zXXuk899VQeffTR\n5lqcmVlZKebpqhsTVGuAvsDYPPONjIh3m3PFw4cPp7q6ujkXaWZWNoqyxVAvQXVCRDwL1DXX8jem\nqxaSsGpmZpunKFsMEXGOpCNoemsggD9JCuC/I+LGfDPWC9HjsiHrgEyWSUPefvttampq8r7fFq1Z\ns6asft7N4bHJz2OTX7mOTdpZSYdExPLkBj2PSXopIp5oaMakadwI0Kf/gLhmYab06gkjGlxwdXU1\nnTt3bjMBWIVoS4Ffzc1jk5/HJr9yHZu0s5KWJ/+ulHQ/cBDQYGPI1qlje5Y0cBMeMzPbeqmdriqp\ns6TtNz4HRgPNEqA3fvx4Dj74YJYsWULv3r255ZZbmmOxZmZloehbDJJ2A+YBXYENki4C9ga6A/cn\n91zuAPwuIprlHNO77rqrORZjZlaWitYYshNUgd4NzPIRsG+x1m9mZlumTV75bGZmW86NwczMcrgx\nmJlZDjcGMzPL4cZgZmY53BjMzCyHG4OZmeVwYzAzsxyKiLRr2GySVgNL0q6jBHUHmvXeFm2IxyY/\nj01+bWls+kbELoXMmHa66pZaEhEHpl1EqZE0z+PSMI9Nfh6b/Mp1bLwryczMcrgxmJlZjtbaGPLe\n6a3MeVzy89jk57HJryzHplUefDYzs+JprVsMZmZWJG4MZmaWo1U1BklHSFoi6VVJk9KuJ02Svihp\ntqRFkl6UdGEyfSdJj0l6Jfl3x7RrTYOk9pKek/RQ8noPSX9Lvju/l/SFtGtMg6Ruku6V9JKkxZIO\n9ncmQ9J/JP8vvSDpLknbluv3ptU0BkntgRuAI8ncGnS8pL3TrSpV64DvRsTewFeAbyXjMQmYFRED\ngVnJ63J0IbA46/XPgKkRMQB4HzgjlarSdx3waETsSeYOiovxdwZJvYALgAMjohJoD5xEmX5vWk1j\nAA4CXo2I1yLiU2A6cGzKNaUmIt6KiL8nz1eT+R+8F5kxmZbMNg0Ym06F6ZHUGxgD3Jy8FvA14N5k\nlnIdlx2A4cAtABHxaUR8gL8zG3UAOknqAGwHvEWZfm9aU2PoBSzNer0smVb2JPUD9gP+BuwaEW8l\nb70N7JpSWWm6FvgesCF5vTPwQUSsS16X63dnD+Ad4LZkN9vNkjrj7wwRsRy4GniDTEP4EJhPmX5v\nWlNjsAZI6gLcB1wUER9lvxeZc5HL6nxkSUcDKyNiftq1lKAOwP7AryJiP6CGeruNyvE7A5AcVzmW\nTPPsCXQGjki1qBS1psawHPhi1uveybSyJakjmaZwZ0TMSCavkLR78v7uwMq06kvJV4F/kVRNZnfj\n18jsV++W7CKA8v3uLAOWRcTfktf3kmkU5f6dAfhn4PWIeCci6oAZZL5LZfm9aU2N4VlgYHKWwBfI\nHBh6MOWaUpPsN78FWBwRv8h660HglOT5KcDMlq4tTRHx/YjoHRH9yHxHHo+ICcBs4PhktrIbF4CI\neBtYKqkimXQYsIgy/84k3gC+Imm75P+tjWNTlt+bVnXls6SjyOw/bg/cGhE/Sbmk1Eg6BHgSWMhn\n+9J/QOY4w91AH+AfwAkR8V4qRaZM0ghgYkQcLak/mS2InYDngG9GxCdp1pcGSUPJHJT/AvAacBqZ\nPxDL/jsjaTJwIpkz/p4DziRzTKHsvjetqjGYmVnxtaZdSWZm1gLcGMzMLIcbg5mZ5XBjMDOzHG4M\nZmaWo0PTs5iVB0nryZz+u9HYiKhOqRyz1Ph0VbOEpDUR0aUF19chK4fHrGR4V5JZgSTtLukJSVVJ\nZv8/JdOPkPR3SQskzUqm7STpAUnPS5oraZ9k+hWSbpT0J+C3yX0jfi7p2WTes1P8Ec0A70oyy9ZJ\nUlXy/PWIGFfv/W8Af4yInyT3B9lO0i7ATcDwiHhd0k7JvJOB5yJirKSvAb8FhibvHQAcEhG1ks4C\nPoyIL0vaBnhK0p8i4vVi/qBmjXFjMPtMbUQMbeT9Z4Fbk/DCByKiKondeGLjL/KsKIlDgK8n0x6X\ntLOkrsl7D0ZEbfJ8NLCPpI15PDsAAwE3BkuNG4NZgSLiCUnDydwE6HZJPydzV6/NVZP1XMD5EfHH\n5qjRrDn4GINZgST1BVZExE1kkm33B+YCwyXtkcyzcVfSk8CEZNoI4N3698tI/BE4N9kKQdKg5OY5\nZqnxFoNZ4UYAF0uqA9YAJ0fEO8lxghmS2pG5l8Eo4Aoyu52eB9byWax1fTcD/YC/J3HP71Amt4+0\n0uXTVc3MLId3JZmZWQ43BjMzy+HGYGZmOdwYzMwshxuDmZnlcGMwM7McbgxmZpbj/wPqxUPxroT/\n9AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x2341db399b0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "plt.bar(range(len(model.feature_importances_)), model.feature_importances_)\n",
    "plt.xlabel(\"Features\")\n",
    "plt.xticks(np.arange(X_train_std.shape[1]+1), feature_names)\n",
    "plt.ylabel(\"Importance score\")\n",
    "plt.show()\n",
    "print(\"Scores using XGBoost's function:\")\n",
    "xgb.plot_importance(model, ylabel = 'Features')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Feature Selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.        ,  0.00226244,  0.00678733,  0.0158371 ,  0.01809955,\n",
       "        0.02036199,  0.02941176,  0.0361991 ,  0.0361991 ,  0.06561086,\n",
       "        0.09049774,  0.09502263,  0.11538462,  0.11538462,  0.15837105,\n",
       "        0.19457014], dtype=float32)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Fit model using each importance as a threshold\n",
    "thresholds = sort(model.feature_importances_)\n",
    "thresholds"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## On training set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top 16 features selected\n",
      "Thresh=0.000, n_features=16, Accuracy: 50.07%\n",
      "  \n",
      "Top 15 features selected\n",
      "Thresh=0.002, n_features=15, Accuracy: 50.07%\n",
      "  \n",
      "Top 14 features selected\n",
      "Thresh=0.007, n_features=14, Accuracy: 50.07%\n",
      "  \n",
      "Top 13 features selected\n",
      "Thresh=0.016, n_features=13, Accuracy: 50.05%\n",
      "  \n",
      "Top 12 features selected\n",
      "Thresh=0.018, n_features=12, Accuracy: 50.10%\n",
      "  \n",
      "Top 11 features selected\n",
      "Thresh=0.020, n_features=11, Accuracy: 50.15%\n",
      "  \n",
      "Top 10 features selected\n",
      "Thresh=0.029, n_features=10, Accuracy: 50.08%\n",
      "  \n",
      "Top 9 features selected\n",
      "Thresh=0.036, n_features=9, Accuracy: 50.13%\n",
      "  \n",
      "Top 9 features selected\n",
      "Thresh=0.036, n_features=9, Accuracy: 50.13%\n",
      "  \n",
      "Top 7 features selected\n",
      "Thresh=0.066, n_features=7, Accuracy: 49.99%\n",
      "  \n",
      "Top 6 features selected\n",
      "Thresh=0.090, n_features=6, Accuracy: 48.70%\n",
      "  \n",
      "Top 5 features selected\n",
      "Thresh=0.095, n_features=5, Accuracy: 48.18%\n",
      "  \n",
      "Top 4 features selected\n",
      "Thresh=0.115, n_features=4, Accuracy: 48.16%\n",
      "  \n",
      "Top 4 features selected\n",
      "Thresh=0.115, n_features=4, Accuracy: 48.16%\n",
      "  \n",
      "Top 2 features selected\n",
      "Thresh=0.158, n_features=2, Accuracy: 42.41%\n",
      "  \n",
      "Top 1 features selected\n",
      "Thresh=0.195, n_features=1, Accuracy: 31.14%\n",
      "  \n"
     ]
    }
   ],
   "source": [
    "train_accs = []\n",
    "num_features = []\n",
    "thresh_vals = []\n",
    "\n",
    "for thresh in thresholds:\n",
    "    \n",
    "    # select features using threshold\n",
    "    selection = SelectFromModel(model, threshold=thresh, prefit=True)\n",
    "    #selecting features based on importance weights\n",
    "    #features whose importance is >= thresh are kept, and the remaining discarded\n",
    "    \n",
    "    select_X_train = selection.transform(X_train_std)\n",
    "    n_features = select_X_train.shape[1]\n",
    "    print(\"Top {} features selected\".format(n_features))\n",
    "    selection_model = XGBClassifier()\n",
    "    selection_model.fit(select_X_train, y_train)\n",
    "    \n",
    "    #Evaluate model on training set\n",
    "    y_pred = selection_model.predict(select_X_train)\n",
    "    predictions = [round(value) for value in y_pred]\n",
    "    accuracy = accuracy_score(y_train, predictions)\n",
    "    print(\"Thresh=%.3f, n_features=%d, Accuracy: %.2f%%\" % (thresh, n_features, accuracy*100.0))\n",
    "    print(\"  \")\n",
    "    train_accs.append(accuracy)\n",
    "    num_features.append(n_features)\n",
    "    thresh_vals.append(thresh)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    Threshold  Training accuracy  n_features\n",
      "5    0.020362           0.501513          11\n",
      "7    0.036199           0.501300           9\n",
      "8    0.036199           0.501300           9\n",
      "4    0.018100           0.501038          12\n",
      "6    0.029412           0.500750          10\n",
      "0    0.000000           0.500700          16\n",
      "1    0.002262           0.500700          15\n",
      "2    0.006787           0.500700          14\n",
      "3    0.015837           0.500500          13\n",
      "9    0.065611           0.499925           7\n",
      "10   0.090498           0.486988           6\n",
      "11   0.095023           0.481750           5\n",
      "12   0.115385           0.481637           4\n",
      "13   0.115385           0.481637           4\n",
      "14   0.158371           0.424088           2\n",
      "15   0.194570           0.311350           1\n",
      "Best training accuracy is obtained upon selection of top 11 features (according to importance score)\n",
      "The corresponding threshold value is 0.020361991599202156\n"
     ]
    }
   ],
   "source": [
    "df = pd.DataFrame(data= {'Threshold':thresh_vals,'n_features':num_features, 'Training accuracy':train_accs})\n",
    "sorted_df = df.sort_values(by = ['Training accuracy'], ascending = False)\n",
    "print(sorted_df)\n",
    "sorted_df.reset_index(inplace = True) #reset because sorting messed up the indices\n",
    "idx = sorted_df['Training accuracy'].idxmax()\n",
    "sel_row= sorted_df.iloc[idx]\n",
    "best_features = int(sel_row['n_features'])\n",
    "best_thresh = sel_row['Threshold']\n",
    "print('Best training accuracy is obtained upon selection of top {} features (according to importance score)'.format(best_features))\n",
    "print(\"The corresponding threshold value is {}\".format(best_thresh))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Modify training and test data to include only selected features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top 11 features selected\n",
      "New training set (80000, 11) and labels (80000,)\n",
      "New test set (4000, 11) and labels (4000,) for each SNR value\n"
     ]
    }
   ],
   "source": [
    "X_test_new = defaultdict(list)\n",
    "\n",
    "selection = SelectFromModel(model, threshold = best_thresh, prefit=True)\n",
    "\n",
    "X_train_new  = selection.transform(X_train_std)\n",
    "n_features = select_X_train.shape[1]\n",
    "print(\"Top {} features selected\".format(best_features))\n",
    "print(\"New training set {} and labels {}\".format(X_train_new.shape, y_train.shape))\n",
    "\n",
    "for snr in snrs:\n",
    "    X_test_new[snr] = selection.transform(X_test_std[snr])\n",
    "print(\"New test set {} and labels {} for each SNR value\".format(X_test_new[10].shape, y_test[10].shape))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Store the new training and test sets in Jupyter's db"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stored 'X_train_new' (ndarray)\n",
      "Stored 'X_test_new' (defaultdict)\n",
      "Stored variables and their in-db values:\n",
      "X_16_val                  -> array([[ 0.10924883,  1.83030605, -0.14807631, ...\n",
      "X_32_val                  -> array([[ 0.66944195,  0.46536115,  0.79919788, ...\n",
      "X_32test_std              -> defaultdict(<class 'list'>, {0: array([[ 0.6694419\n",
      "X_32train_std             -> array([[-0.74031227,  0.0126481 , -0.30967801, ...\n",
      "X_test                    -> defaultdict(<class 'list'>, {0: array([[[ -6.40490\n",
      "X_test_new                -> defaultdict(<class 'list'>, {0: array([[ 0.1092488\n",
      "X_test_std                -> defaultdict(<class 'list'>, {0: array([[ 0.1092488\n",
      "X_train                   -> array([[[ 0.00119031,  0.00873315,  0.00641749, ..\n",
      "X_train_new               -> array([[-0.74031227,  0.23616372, -0.18182195, ...\n",
      "X_train_std               -> array([[-0.74031227,  0.0126481 , -0.30967801, ...\n",
      "snrs                      -> [-20, -18, -16, -14, -12, -10, -8, -6, -4, -2, 0, \n",
      "y_16_val                  -> array([6, 6, 5, ..., 0, 4, 1])\n",
      "y_32_test                 -> defaultdict(<class 'list'>, {0: array([2, 2, 4, ..\n",
      "y_32_train                -> array([0, 3, 4, ..., 0, 3, 1])\n",
      "y_32_val                  -> array([2, 2, 4, ..., 0, 7, 3])\n",
      "y_test                    -> defaultdict(<class 'list'>, {0: array([6, 6, 5, ..\n",
      "y_train                   -> array([0, 3, 4, ..., 0, 3, 1])\n"
     ]
    }
   ],
   "source": [
    "%store X_train_new\n",
    "%store X_test_new\n",
    "%store"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
